package lion.crawl.fetch;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.Map;
import java.util.zip.GZIPInputStream;

import lion.dev.io.FileUtil;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * @author Helin
 * @date 2012-8-12
 * @desc 下载
 */
public class HttpDownloader implements IDownloader {

	private static Log log = LogFactory.getLog(HttpDownloader.class);

	private Map<String, String> headers = new HashMap<>();

	@Override
	public Content fatch(String url) throws IOException {

		URL theurl = new URL(url);
		log.debug("download:" + url);

		URLConnection conn = theurl.openConnection();
		conn.addRequestProperty("User-Agent", "	Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1");
		conn.addRequestProperty("Accept-Encoding", "gzip, deflate");

		for (String key : headers.keySet()) {
			conn.addRequestProperty(key, headers.get(key));
		}

		InputStream is = conn.getInputStream();
		Content content = new Content();
		for (String key : conn.getHeaderFields().keySet()) {
			content.setHeader(key, conn.getHeaderField(key));
		}
		// GZIP
		String ce = content.getHeader("Content-Encoding");
		if (StringUtils.isNotBlank(ce) && ce.indexOf("gzip") > -1) {
			is = new GZIPInputStream(is);
		}
		content.setContent(FileUtil.readBytes(is));

		return content;
	}

	@Override
	public String curl(String url) throws IOException {

		return FileUtil.read(fatch(url).getContent());
	}

	public void setHeader(String key, String value) {

		headers.put(key, value);
	}
}
